rm(list=ls())
#备份数据
save(list=ls(),file="data20131111")


#解压.csv数据
#z:\licaiData\index_csv\sh_index\
#for %a in (\licaiData\index_csv\sh_index\*.rar) do "C:\Program Files (x86)\WinRAR\Rar.exe" e %a 000001*.csv C:\R\data\000001\

#读入股票数据，写入save/stk.data.inter.intra4 文件中
read.stk<-function(){
  #获取日内数据
  Stock.intra1<-read.csv("stk_intra_BySTK_2005_2011年9月/000001_intra.csv", colClasses=c(NA,NA,NA,rep('NULL',6),NA,rep('NULL',2)), stringsAsFactors=FALSE)
  Stock.intra2<-read.csv("000002_intra.csv", colClasses=c(NA,NA,NA,rep('NULL',6),NA,rep('NULL',2)), stringsAsFactors=FALSE)
  Stock.intra4<-read.csv("000004_intra.csv", colClasses=c(NA,NA,NA,rep('NULL',6),NA,rep('NULL',2)), stringsAsFactors=FALSE)
  Stock.intra5<-read.csv("000004_intra.csv", colClasses=c(NA,NA,NA,rep('NULL',6),NA,rep('NULL',2)), stringsAsFactors=FALSE)
  
  wd = getwd()
  setwd('stk_intra_BySTK_2005_2011年9月/')
  files = dir()	
  Stock.intra.list = list() #以股票的symbol为index
  #for(i in (1:length(files)))
  for(i in (1:2))
  {
	Stock.intra<-read.csv(files[i], colClasses=c(NA,NA,NA,rep('NULL',6),NA,rep('NULL',2)), stringsAsFactors=FALSE)
	Stock.intra.list[[as.character(Stock.intra$symbol[1])]] = Stock.intra
  }
  setwd(wd) 	
  
  #获取日间数据
  load("../allstocktill20130715.RData")
  Stock.inter = allstocktill20130715[,c('time','symbol','CLOSE')]
  Stock.inter.list <<- split(Stock.inter,Stock.inter$symbol); #将股票将日内数据按照日期分割成list
  
  save(Stock.intra.list, Stock.inter.list, file='save/stk.data.inter.intra4')
}

#处理股票日内，获取统计数据
#读入矩阵数据，输出统计list
sta.stock<- function(Stock.intra.list){
  Stock.symbol = names(Stock.intra.list)
  #将所有股票的日期汇总
  Stock.date = c()
  for(i in 1:length(Stock.symbol)){
	Stock.date = sort(unique(c(Stock.date, Stock.intra.list[[Stock.symbol[i]]]$date)))
  }
  #Stock.date = as.Date(as.character(Stock.date),"%Y%m%d") 
  
  #产生 指数每日波动数据统计量
  sta.list <- matrix(nrow=length(Stock.date), ncol=length(Stock.symbol))
  colnames(sta.list) <- Stock.symbol
  rownames(sta.list) <- as.Date(Stock.date,"%Y%m%d")
  for(i in 1:length(Stock.symbol)){
	print(i)
	Stock.intra <- split(Stock.intra.list[[Stock.symbol[i]]],Stock.intra.list[[Stock.symbol[i]]]$date); #按照日期分割日内数据
	Stock.date.symbol = names(Stock.intra)
	for (j in 1:length(Stock.date.symbol))
	{
		print(j)
		Stock.oneday = Stock.intra[[Stock.date.symbol[j]]]
		Stock.oneday.price = Stock.oneday$CLOSE;
		
		if(sum(Stock.oneday.price)==0) #数据全部缺失,利用上个值填充
		{
			wave.oneday = NA
		} else{
		if(length(Stock.oneday.price)<2) #可能是停盘
			wave.oneday = list(fluc.mean=prev.fluc.mean, fluc.sd=prev.fluc.sd, width.mean=prev.width.mean, width.sd=prev.width.sd) #设置为涨幅前一日，此处需要修正
		else
			wave.oneday = StatFun(Stock.oneday.price)
		}
		
		sta.list[as.Date(Stock.date.symbol[j],"%Y%m%d"),Stock.symbol[i]] = prev.fluc.mean = wave.oneday$fluc.mean
	}
  }
  return(sta.list)
}

#生成12日ema
#输入为矩阵，输出为矩阵
ema.12 <- function(data){
  sum = 0 
  ema = matrix(nrow=nrow(data), ncol=ncol(data),dimnames=list(as.character(as.Date(rownames(data),'%Y%m%d')),colnames(data)))
  
  ema[1,] = data[1,]
  for (i in 2:nrow(data))
  {
	data.one.row = data[i,]
	data.last.row = data[i-1,]
	data.last.row[is.na(data.last.row)] = data.one.row[is.na(data.last.row)] # 用上一个值弥补当前NA值
	ema[i,] = 7/13*data[i,] + 6/13*data[i,] 
  }
  return(ema)
}

#读入指数数据，写入save/stk.data.inter.intra4 文件中
read.idx<-function(){
  #合并两年的指数数据
  sh300.2006 = read.csv("index\\sh000300_2006.csv", colClasses=c('NULL',NA,NA,NA,rep('NULL',15)), stringsAsFactors=FALSE) #只读取 证券代码 时间 最新 
  sh300.2007 = read.csv("index\\sh000300_2007.csv", colClasses=c('NULL',NA,NA,NA,rep('NULL',15)), stringsAsFactors=FALSE) #只读取 证券代码 时间 最新 
  sh300.2008 = read.csv("index\\sh000300_2008.csv", colClasses=c('NULL',NA,NA,NA,rep('NULL',15)), stringsAsFactors=FALSE) #只读取 证券代码 时间 最新 
  sh300.2009 = read.csv("index\\sh000300_2009.csv", colClasses=c('NULL',NA,NA,NA,rep('NULL',15)), stringsAsFactors=FALSE) #只读取 证券代码 时间 最新 
  idx= rbind(sh300.2006, sh300.2007, sh300.2008, sh300.2009)
  rm(sh300.2006, sh300.2007, sh300.2008, sh300.2009)
  
  #将时间拆分，然后绑定在一起
  idx.time.2 = do.call('rbind',strsplit(idx$时间,' ', fixed=TRUE)) ; 
  idx$date = idx.time.2[,1]
  idx$time = idx.time.2[,2]
  idx.intra.list <<- split(idx.intra.list, idx.intra.list$date); #将指数数据按日期分割
  
  #日间数据
  load('csi.RData')
  idx.inter = csi
  save(idx, idx.inter, file='save/idx.intra.inter')
}

#获取沪深300日内数据,比如"index\\sh000300_2006.csv"
sta.idx<- function(idx.intra.list){
  #产生 指数每日波动数据统计量
  idx.intra.list.date <- names(idx.intra.list)
  Date.length <- length(idx.intra.list.date)
  idx.sta <- list()
  for(i in 1:(Date.length)){
    print(i)
    idx.sz1.oneday = idx.intra.list[[idx.intra.list.date[i]]];
    idx.sz1.price = idx.sz1.oneday$最新;
    
    if(sum(idx.sz1.price)==0) #数据全部缺失,利用上个值填充
    {
      wave.oneday = list(fluc.mean=prev.fluc.mean, fluc.sd=prev.fluc.sd, width.mean=prev.width.mean, width.sd=prev.width.sd) #设置为涨幅无穷大
    } else{
      idx.wave.oneday = StatFun(idx.sz1.price)
    }
    
    idx.sta[[idx.intra.list.date[i]]]$fluc.mean = prev.fluc.mean = idx.wave.oneday$fluc.mean
    idx.sta[[idx.intra.list.date[i]]]$fluc.sd = prev.fluc.sd = idx.wave.oneday$fluc.sd
    idx.sta[[idx.intra.list.date[i]]]$width.mean = prev.width.mean = idx.wave.oneday$width.mean
    idx.sta[[idx.intra.list.date[i]]]$width.sd = prev.width.sd = idx.wave.oneday$width.sd
  }
  return(idx.sta)
}

#读入指数 csv数据
readcsv.no.filter<-function(path){
  #读取日内数据
  #path = "D:/SVN_code/R/script"; 
  path.old = getwd(1);
  setwd(path);
  files=dir(); #当前目录所有文件
  Stock<-list();
  #读取多个文件
  for(i in (1:length(files))){
    #for(i in (1:4)){  
    Stock.oneday<-try(read.csv(paste("",files[i],sep=''), stringsAsFactors=FALSE), silent=TRUE);  #读入CSV的时候不要将变量变成factor
    Stock.oneday<-Stock.oneday[,c(1,2,3,4,6,7,8,15,18)]; #提取0的列
    colnames(Stock.oneday)<-c("Market","symbol", "time","price","close.amount","close.pos","direction","buy1","sell1");
    Stock.date<-strsplit(Stock.oneday[1,3],' ')[[1]][1]; #提取日期
    Stock.oneday[,3]<-gsub(Stock.date,'',Stock.oneday[,3]);
    Stock[[Stock.date]] = Stock.oneday;
  }
  
  setwd(path.old); #返回原路径
  return(Stock);
}

#统计股票日内波动，生成波动量和波动宽度
StatFun<-function(Data){
  #数据预处理，消除NA数据和0数据
  for (j in 2:(length(Data)))  #向前填充，需要第一个值非0
  {
    if(Data[j]==0)
      Data[j] = Data[j-1]; 
  }
  for (j in (length(Data)-1):1) #向后填充，需要最后一个值非0
  {
    if(Data[j]==0)
      Data[j] = Data[j+1]; 
  }
  Data = Data[Data!=0]  #消除0数据
  
  #统计Data的drawdown
  PPos<-c();  #上一个极大值
  NPos<-c();  #上一个极小值
  LstPos<-1;
  #波动和宽度
  wave<- list()
  k <- 1;
  if(length(Data)>2) #每日至少三个收盘价
  {
    for(i in 2:(length(Data)-1)){
      #极大值 (大于上一个极小值，并小于下一个值)
      if(Data[i]>Data[LstPos]&& Data[i]>Data[i+1]){  
        #wave<-rbind(wave,c((Data[i]-Data[LstPos])/Data[LstPos],i-LstPos));    
        wave[[as.character(k)]]$Fluc = (Data[i]-Data[LstPos])/Data[LstPos]
        wave[[as.character(k)]]$Width = i-LstPos
        k = k+1
        LstPos<-i;      
      }
      #极小值 (小于上一个极大值,并大于下一个值)
      if((Data[i]<Data[LstPos])&& (Data[i]<Data[i+1])){  
        #wave<-rbind(wave,c((Data[i]-Data[LstPos])/Data[LstPos],i-LstPos)); 
        wave[[as.character(k)]]$Fluc = (Data[i]-Data[LstPos])/Data[LstPos]
        wave[[as.character(k)]]$Width = i-LstPos
        k = k+1
        LstPos<-i;
      }
    }
  } else
  {
    LstPos=1; #将最初值设置为第一个，最后一个值设置为最终值
    i=length(Data);    
  }
  
  #最后一个值
  #wave<-rbind(wave,c((Data[i]-Data[LstPos])/Data[LstPos],i-LstPos));
  #wave<-wave[-1,];  #删除第一行N
  wave[[as.character(k)]]$Fluc = (Data[i]-Data[LstPos])/Data[LstPos]
  wave[[as.character(k)]]$Width = i-LstPos
  k = k+1
  
  fluc = sapply(wave,"[[","Fluc",simplify=TRUE) #将波动幅度提取出来作为一个矩阵
  fluc.mean = mean(fluc)
  fluc.sd = sd(fluc)
  width = sapply(wave,"[[","Width",simplify=TRUE)
  width.mean = mean(width)
  width.sd = sd(width)
  
  wave.summary = list(fluc.mean=fluc.mean, fluc.sd=fluc.sd, width.mean=width.mean, width.sd=width.sd)
  return(wave.summary);
}

#准备股票数据
data.pre <- function(Stock.intra.list, Stock.inter.list){  
  #获取日内数据
  Stock.symbol=Stock.intra$symbol[1];
  Stock.intra.his <<- Stock.intra[Stock.intra$date<"20051231",]; #将2005-12-31前的数据用来训练
  Stock.intra.test <<- Stock.intra[Stock.intra$date>="20051231",];
  
  #获取指数日内数据，只有2013年
  #idx.sz1.intra = readcsv.no.filter("000001");
  #View(idx.sz1.intra[c(1,2,3,4),]); #看非0的列
  
  #获取历史数据
  Stock.intra.his.date <<- unique(as.Date(as.character(Stock.intra.his$date),"%Y%m%d"));    #提取日内股票价格日期序列
  Stock.intra.his.list <<- split(Stock.intra.his,Stock.intra.his$date); #按照日期分割日内数据 
  
  #获取测试数据
  Stock.intra.test.date <<- unique(as.Date(as.character(Stock.intra.test$date),"%Y%m%d"));    #提取日内股票价格日期序列
  Stock.intra.test.list <<- split(Stock.intra.test,Stock.intra.test$date); #按照日期分割日内数据 
  
  Stock.inter.One.Stock = Stock.inter.list[[Stock.symbol]]; #选出该股票的日间数据
  Stock.inter.date = as.Date(as.character(Stock.inter.One.Stock$time),"%Y-%m-%d");    #提取日间股票价格日期序列
  
  #选取历史日间数据
  date.sel=match(Stock.intra.his.date, Stock.inter.date); #选出与日内数据日期匹配的行
  Stock.inter.his <<- Stock.inter.One.Stock[date.sel[!is.na(date.sel)],];
  
  #选取测试日间数据
  date.sel=match(Stock.intra.test.date, Stock.inter.date); #选出与日内数据日期匹配的行
  Stock.inter.test <<- Stock.inter.One.Stock[date.sel[!is.na(date.sel)],];
}

#处理历史股票数据得到统计波动幅度和宽度统计量
Stock.his.sta<-function(Stock.intra.his.date, Stock.intra.his.list){
  Stock.wave<-list();
  Stock.wave.all<-list();
  
  i=1
  Date.string= gsub('-','',as.character(Stock.intra.his.date[i])) #将Date格式转换为字符串格式
  Stock.wave= StatFun(Stock.intra.his.list[[Date.string]]$CLOSE);  
  Stock.wave.all = Stock.wave;
  
  for(i in 2:length(Stock.intra.his.date))
    #for(i in 1:2)
  {
    Date.string= gsub('-','',as.character(Stock.intra.his.date[i])) #将Date格式转换为字符串格式
    Stock.wave= StatFun(Stock.intra.his.list[[Date.string]]$CLOSE);  
    if(!is.na(Stock.wave$fluc.sd))
      Stock.wave.all = mapply(cbind, Stock.wave.all, Stock.wave, SIMPLIFY = FALSE);
  }
  
  #计算波动的统计特性，也可以考虑用其他方法，而不是正态分布拟合
  Stock.wave.fluc.mean<<-mean(Stock.wave.all$fluc.mean);
  Stock.wave.fluc.sd<<-mean(Stock.wave.all$fluc.sd); #日间波动
  Stock.wave.width.mean<<-mean(Stock.wave.all$width.mean);
  Stock.wave.width.sd<<-mean(Stock.wave.all$width.sd); #日间波动
  
  #绘图看一下，波动宽度和幅度的统计特性
  #hist(Stock.wave.all[,"Width"])
  #hist(Stock.wave.all[,"Fluc"])
  
  #设置正常条件和触发条件，是不是可以将波动率按照指数移动平均方式统计
  Norm.fluc = Stock.wave.fluc.mean+1*Stock.wave.fluc.sd; #正常股票波动量,2*delta
  Trig.fluc.pos  = Stock.wave.fluc.mean+1*Stock.wave.fluc.sd; #买入触发股票波动量,4*delta
  Trig.fluc.neg  = Stock.wave.fluc.mean-1*Stock.wave.fluc.sd; #卖出触发股票波动量,4*delta
  Norm.width = Stock.wave.width.mean+1*Stock.wave.width.sd; #正常股票波动量,2*delta
  Trig.width  = Stock.wave.width.mean+1*Stock.wave.width.sd; #触发股票波动量,4*delta
  
  Stock.his.sta <- list(Norm.fluc=Norm.fluc, Trig.fluc.pos=Trig.fluc.pos, Trig.fluc.neg=Trig.fluc.neg, Norm.width=Norm.width, Trig.width=Trig.width)
  return(Stock.his.sta)
}



###########################################################################################
###建立策略

#准备日间数据

read.idx()

#处理日内数据的波动
data.pre(Stock.intra2, Stock.inter.list)

idx.sta <- sta.idx(idx.intra.list) #统计指数历史波动
save(idx.sta, file="save/idx.sta")
idx.ema.12 <- ema.12(idx.sta)

#Stock.his.sta<-Stock.his.sta(Stock.intra.his.date, Stock.intra.his.list) #历史股票波动统计

read.stk()
Stock.sta.list <- sta.stock(Stock.intra.list) #准备股票历史波动
Stock.ema.12 <- ema.12(Stock.sta.list)

#测试日期列表
final.date = "2009-12-31"  
test.date = "2006-12-31"  
Stock.symbol = colnames(Stock.ema.12)
#Stock.intra.test.date = as.Date(names(Stock.intra.test.list), "%Y%m%d");


#绘制一些图形
par(mfrow=c(1,1))
buy.bar = barplot(buy.summary$buy.signal)  #绘制买入/卖出的柱状图
lines(buy.summary$Stock.pos/2,col="blue")
lines(Stock.inter.test$CLOSE/max(Stock.inter.test$CLOSE),col="yellow") #归一化后的股票价格

#绘制一些图形
View(buy.summary)
par(mfrow=c(1,1))
Stock.inter.test.plot = Stock.inter.test[Stock.inter.test$time<final.date,];
regress = max(Stock.inter.test.plot$CLOSE)/(max(buy.summary$equity.all)/100000)
plot(buy.summary$equity.all/100000, type='l',col="orange")
lines(Stock.inter.test.plot$CLOSE/regress,col="brown")  #绘制买入/卖出的柱状图
lines(buy.summary$Stock.pos/2,col="blue")

plot(Stock.ema.12$fluc.mean, type='l',col="orange")

par(mfrow=c(2,1))
plot(Stock.inter.test.plot$CLOSE/max(Stock.inter.test.plot$CLOSE), type='l',col="orange", main="Stock price")
lines(buy.summary$Stock.fluc.mean/max(buy.summary$Stock.fluc.mean), type='h',col="brown", main="Stock intra fluc")  #绘制买入/卖出的柱状图
plot(buy.summary$idx.fluc.mean/max(buy.summary$idx.fluc.mean), type='h',col="blue",  main="idx intra fluc")

#根据信号强弱选择持有天数，固定持有一定天数
#为何在400日左右的上涨未被检测到

#无法处理震荡下跌情况
#复权因子